from __future__ import division
from operator import itemgetter

queryTermFreqDistributionDict = {}
# queryTermProbabilityDistributionDict = {}
queryTermProbabilityDistributionList = []
queryDict = {}
totalProbability = 0.0

totalNumberOfQueryTerms = 0

inputQueryFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/100KQueries"
inputQueryHandler = open(inputQueryFileName,"r")

outputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/queryTermProbabilityDistribution_sortedByTerm"
outputFileHandler1 = open(outputFileName1,"w")

outputFileName2 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/queryTermProbabilityDistribution_sortedByProbability"
outputFileHandler2 = open(outputFileName2,"w")




for line in inputQueryHandler.readlines():
    elements = line.strip().split(":")
    queryID = int(elements[0])
    
    data = elements[1]
    data = data.lower()
    
    for i in range(0,len(data)):
        # print "data[i]:",ord(data[i])
        if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
            # Just replace them with a space.
            data = data[:i] + " " + data[i+1:]
    
    queryContent = data

    if queryID not in queryDict:
        queryDict[queryID] = queryContent
    
    queryTerms = queryContent.strip().split(" ")
    
    totalNumberOfQueryTerms += len(queryTerms)
    
    for queryTerm in queryTerms:
        if queryTerm.strip() != "":
            if queryTerm not in queryTermFreqDistributionDict:
                queryTermFreqDistributionDict[queryTerm] = 1
            else:
                queryTermFreqDistributionDict[queryTerm] += 1
        else:
            # because queryTerm's content is an empty string
            pass

for queryTerm in queryTermFreqDistributionDict:
    currentProbability = queryTermFreqDistributionDict[queryTerm] / totalNumberOfQueryTerms
    tuple = (queryTerm,currentProbability)
    totalProbability += currentProbability
    queryTermProbabilityDistributionList.append(tuple)

print "totalProbability:",totalProbability

queryTermProbabilityDistributionList.sort(cmp=None, key=itemgetter(0), reverse=False)
for element in queryTermProbabilityDistributionList:
    (queryTerm,probability) = element 
    outputFileHandler1.write(queryTerm + " " + str(probability) + "\n")
   
queryTermProbabilityDistributionList.sort(cmp=None, key=itemgetter(1), reverse=True)
for element in queryTermProbabilityDistributionList:
    (queryTerm,probability) = element 
    outputFileHandler2.write(queryTerm + " " + str(probability) + "\n")

print "----->","len(queryDict):",len(queryDict)
print "----->","len(queryTermFreqDistributionDict):",len(queryTermFreqDistributionDict)

inputQueryHandler.close()
outputFileHandler1.close()
outputFileHandler2.close()

